{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Lab 8.3.3 Bagging and Random Forests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-09-19T12:27:19.208719Z",
     "start_time": "2023-09-19T12:27:16.609784Z"
    }
   },
   "outputs": [
    {
     "ename": "ImportError",
     "evalue": "\n`load_boston` has been removed from scikit-learn since version 1.2.\n\nThe Boston housing prices dataset has an ethical problem: as\ninvestigated in [1], the authors of this dataset engineered a\nnon-invertible variable \"B\" assuming that racial self-segregation had a\npositive impact on house prices [2]. Furthermore the goal of the\nresearch that led to the creation of this dataset was to study the\nimpact of air quality but it did not give adequate demonstration of the\nvalidity of this assumption.\n\nThe scikit-learn maintainers therefore strongly discourage the use of\nthis dataset unless the purpose of the code is to study and educate\nabout ethical issues in data science and machine learning.\n\nIn this special case, you can fetch the dataset from the original\nsource::\n\n    import pandas as pd\n    import numpy as np\n\n    data_url = \"http://lib.stat.cmu.edu/datasets/boston\"\n    raw_df = pd.read_csv(data_url, sep=\"\\s+\", skiprows=22, header=None)\n    data = np.hstack([raw_df.values[::2, :], raw_df.values[1::2, :2]])\n    target = raw_df.values[1::2, 2]\n\nAlternative datasets include the California housing dataset and the\nAmes housing dataset. You can load the datasets as follows::\n\n    from sklearn.datasets import fetch_california_housing\n    housing = fetch_california_housing()\n\nfor the California housing dataset and::\n\n    from sklearn.datasets import fetch_openml\n    housing = fetch_openml(name=\"house_prices\", as_frame=True)\n\nfor the Ames housing dataset.\n\n[1] M Carlisle.\n\"Racist data destruction?\"\n<https://medium.com/@docintangible/racist-data-destruction-113e3eff54a8>\n\n[2] Harrison Jr, David, and Daniel L. Rubinfeld.\n\"Hedonic housing prices and the demand for clean air.\"\nJournal of environmental economics and management 5.1 (1978): 81-102.\n<https://www.researchgate.net/publication/4974606_Hedonic_housing_prices_and_the_demand_for_clean_air>\n",
     "output_type": "error",
     "traceback": [
      "\u001B[0;31m---------------------------------------------------------------------------\u001B[0m",
      "\u001B[0;31mImportError\u001B[0m                               Traceback (most recent call last)",
      "Cell \u001B[0;32mIn[1], line 4\u001B[0m\n\u001B[1;32m      1\u001B[0m \u001B[38;5;28;01mimport\u001B[39;00m \u001B[38;5;21;01mpandas\u001B[39;00m \u001B[38;5;28;01mas\u001B[39;00m \u001B[38;5;21;01mpd\u001B[39;00m\n\u001B[1;32m      2\u001B[0m \u001B[38;5;28;01mimport\u001B[39;00m \u001B[38;5;21;01mnumpy\u001B[39;00m \u001B[38;5;28;01mas\u001B[39;00m \u001B[38;5;21;01mnp\u001B[39;00m\n\u001B[0;32m----> 4\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01msklearn\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mdatasets\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m load_boston\n\u001B[1;32m      5\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01msklearn\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mensemble\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m RandomForestRegressor\n\u001B[1;32m      6\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01msklearn\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mmodel_selection\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m train_test_split,cross_val_score\n",
      "File \u001B[0;32m/opt/anaconda3/envs/py309/lib/python3.9/site-packages/sklearn/datasets/__init__.py:157\u001B[0m, in \u001B[0;36m__getattr__\u001B[0;34m(name)\u001B[0m\n\u001B[1;32m    108\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m name \u001B[38;5;241m==\u001B[39m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mload_boston\u001B[39m\u001B[38;5;124m\"\u001B[39m:\n\u001B[1;32m    109\u001B[0m     msg \u001B[38;5;241m=\u001B[39m textwrap\u001B[38;5;241m.\u001B[39mdedent(\u001B[38;5;124m\"\"\"\u001B[39m\n\u001B[1;32m    110\u001B[0m \u001B[38;5;124m        `load_boston` has been removed from scikit-learn since version 1.2.\u001B[39m\n\u001B[1;32m    111\u001B[0m \n\u001B[0;32m   (...)\u001B[0m\n\u001B[1;32m    155\u001B[0m \u001B[38;5;124m        <https://www.researchgate.net/publication/4974606_Hedonic_housing_prices_and_the_demand_for_clean_air>\u001B[39m\n\u001B[1;32m    156\u001B[0m \u001B[38;5;124m        \u001B[39m\u001B[38;5;124m\"\"\"\u001B[39m)\n\u001B[0;32m--> 157\u001B[0m     \u001B[38;5;28;01mraise\u001B[39;00m \u001B[38;5;167;01mImportError\u001B[39;00m(msg)\n\u001B[1;32m    158\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[1;32m    159\u001B[0m     \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28mglobals\u001B[39m()[name]\n",
      "\u001B[0;31mImportError\u001B[0m: \n`load_boston` has been removed from scikit-learn since version 1.2.\n\nThe Boston housing prices dataset has an ethical problem: as\ninvestigated in [1], the authors of this dataset engineered a\nnon-invertible variable \"B\" assuming that racial self-segregation had a\npositive impact on house prices [2]. Furthermore the goal of the\nresearch that led to the creation of this dataset was to study the\nimpact of air quality but it did not give adequate demonstration of the\nvalidity of this assumption.\n\nThe scikit-learn maintainers therefore strongly discourage the use of\nthis dataset unless the purpose of the code is to study and educate\nabout ethical issues in data science and machine learning.\n\nIn this special case, you can fetch the dataset from the original\nsource::\n\n    import pandas as pd\n    import numpy as np\n\n    data_url = \"http://lib.stat.cmu.edu/datasets/boston\"\n    raw_df = pd.read_csv(data_url, sep=\"\\s+\", skiprows=22, header=None)\n    data = np.hstack([raw_df.values[::2, :], raw_df.values[1::2, :2]])\n    target = raw_df.values[1::2, 2]\n\nAlternative datasets include the California housing dataset and the\nAmes housing dataset. You can load the datasets as follows::\n\n    from sklearn.datasets import fetch_california_housing\n    housing = fetch_california_housing()\n\nfor the California housing dataset and::\n\n    from sklearn.datasets import fetch_openml\n    housing = fetch_openml(name=\"house_prices\", as_frame=True)\n\nfor the Ames housing dataset.\n\n[1] M Carlisle.\n\"Racist data destruction?\"\n<https://medium.com/@docintangible/racist-data-destruction-113e3eff54a8>\n\n[2] Harrison Jr, David, and Daniel L. Rubinfeld.\n\"Hedonic housing prices and the demand for clean air.\"\nJournal of environmental economics and management 5.1 (1978): 81-102.\n<https://www.researchgate.net/publication/4974606_Hedonic_housing_prices_and_the_demand_for_clean_air>\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "from sklearn.datasets import load_boston\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from sklearn.model_selection import train_test_split,cross_val_score\n",
    "\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-09-19T12:27:19.242936Z",
     "start_time": "2023-09-19T12:27:19.211241Z"
    }
   },
   "outputs": [],
   "source": [
    "# loading the dataset\n",
    "boston_data = load_boston()\n",
    "boston = pd.DataFrame(boston_data.data,columns = boston_data['feature_names'])\n",
    "boston['MEDV'] = boston_data['target']\n",
    "boston.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Again, i would advice you to get familier with the data, since its an inbuilt dataset, we can use boston_data.DESCR to get mode information"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- CRIM - per capita crime rate by town\n",
    "- ZN - proportion of residential land zoned for lots over 25,000 sq.ft.\n",
    "- INDUS - proportion of non-retail business acres per town.\n",
    "- CHAS - Charles River dummy variable (1 if tract bounds river; 0 otherwise)\n",
    "- NOX - nitric oxides concentration (parts per 10 million)\n",
    "- RM - average number of rooms per dwelling\n",
    "- AGE - proportion of owner-occupied units built prior to 1940\n",
    "- DIS - weighted distances to five Boston employment centres\n",
    "- RAD - index of accessibility to radial highways\n",
    "- TAX - full-value property-tax rate per $10,000\n",
    "\n",
    "- PTRATIO - pupil-teacher ratio by town\n",
    "- B - 1000(Bk - 0.63)^2 where Bk is the proportion of blacks by town\n",
    "- LSTAT - % lower status of the population\n",
    "- MEDV - Median value of owner-occupied homes in $1000's"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "soucrs - [https://www.cs.toronto.edu/~delve/data/boston/bostonDetail.html]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Its better that we go through the above list once, so that we know what we are dealing with here."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-09-19T12:27:19.212040Z"
    }
   },
   "outputs": [],
   "source": [
    "#splitting the data into train and test \n",
    "X_train,X_test,y_train,y_test = train_test_split(boston.drop('MEDV',axis = 1),boston['MEDV'],test_size = 0.5,random_state = 0)\n",
    "print(X_train.shape,X_test.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Bagging\n",
    "- We have learnt it earlier that random forest is a special case of bagging algorithms, and that random forest uses a subspace of featurs that are considered for each split. Generally random forest considers m features at each split, out of total feature space of p features. \n",
    "- Usually the value of m is sqrt(p), but if m = p, it becomes same as baaging.\n",
    "- Although there is a separate library for bagging in sklearn, we are going to follow the same approach in the book, and use random forests with m = p."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In random forests, we use max_features paramter to control the number of features considered for each split, so, for m = p, we just pass max_features as None, or the size of predictor space - \n",
    "for more on this check out the max_features paramtere from here - https://scikit-learn.org/stable/modules/generated/sklearn.ensemble.RandomForestClassifier.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-09-19T12:27:19.213627Z"
    }
   },
   "outputs": [],
   "source": [
    "bagging = RandomForestRegressor(max_features=None)# or we can pass max_features = len(X_train.columns) = 13\n",
    "bagging.fit(X_train,y_train)\n",
    "print('Training score ',bagging.score(X_train,y_train))\n",
    "print('Test Error ',bagging.score(X_test,y_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here the score means the r2 score\n",
    "So, we can see there is a vast imporvement from the score we were getting the last time when we used decision tree regressor"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### NOTE: In book, the errors are mentioned in RSS, which can be calculated by sklearn.metrics.mean_squared_error than multiplying it by number of observations, i have used R2 score here. (Doesn;t have any effect in comparison though)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Random Forests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-09-19T12:27:19.215212Z"
    }
   },
   "outputs": [],
   "source": [
    "rf = RandomForestRegressor(max_features = 6) # same as book\n",
    "rf.fit(X_train,y_train)\n",
    "print('Training score ',rf.score(X_train,y_train))\n",
    "print('Test Error ',rf.score(X_test,y_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here we are getting more improved results than bagging. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Feature importance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-09-19T12:27:19.215872Z"
    }
   },
   "outputs": [],
   "source": [
    "pd.Series(rf.feature_importances_,index = X_train.columns).sort_values(ascending=False).plot.bar(figsize = (10,6))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "From the above graph we can see that RM and LSTAT are by far the most important features of all."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Happy Learning :)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2023-09-19T12:27:19.216564Z"
    }
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
